{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from keras.layers import Dense, Input, Flatten\n",
    "from keras.layers import GlobalMaxPool1D, Bidirectional, Convolution1D, Embedding, BatchNormalization,MaxPooling1D, Dropout, LSTM\n",
    "from keras import backend as K\n",
    "from keras.engine.topology import Layer\n",
    "from keras import initializers, regularizers, constraints\n",
    "from keras.models import Model\n",
    "from keras.layers.merge import Concatenate\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "INPUT_PATH = '../input/'\n",
    "CACHE_PATH = '../cache/'\n",
    "OUTPUT_PATH ='../output/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data = np.load(CACHE_PATH + 'data.npz')\n",
    "X_train = data['X_train']\n",
    "y_train = data['y_train']\n",
    "X_val = data['X_val']\n",
    "y_val = data['y_val']\n",
    "X_test = data['X_test']\n",
    "embedding_matrix = np.load(CACHE_PATH + 'embedding_matrix.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "MAX_FEATURES = 20000\n",
    "MAX_SEQUENCE_LENGTH = 300\n",
    "EMBEDDING_DIM = 300"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "embedding_layer = Embedding(MAX_FEATURES,\n",
    "                            EMBEDDING_DIM,\n",
    "                            weights=[embedding_matrix],\n",
    "                            input_length=MAX_SEQUENCE_LENGTH,\n",
    "                            trainable=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Attention(Layer):\n",
    "    def __init__(self, step_dim,\n",
    "                 W_regularizer=None, b_regularizer=None,\n",
    "                 W_constraint=None, b_constraint=None,\n",
    "                 bias=True, **kwargs):\n",
    "        \"\"\"\n",
    "        Keras Layer that implements an Attention mechanism for temporal data.\n",
    "        Supports Masking.\n",
    "        Follows the work of Raffel et al. [https://arxiv.org/abs/1512.08756]\n",
    "        # Input shape\n",
    "            3D tensor with shape: `(samples, steps, features)`.\n",
    "        # Output shape\n",
    "            2D tensor with shape: `(samples, features)`.\n",
    "        :param kwargs:\n",
    "        Just put it on top of an RNN Layer (GRU/LSTM/SimpleRNN) with return_sequences=True.\n",
    "        The dimensions are inferred based on the output shape of the RNN.\n",
    "        Example:\n",
    "            model.add(LSTM(64, return_sequences=True))\n",
    "            model.add(Attention())\n",
    "        \"\"\"\n",
    "        self.supports_masking = True\n",
    "        self.init = initializers.get('glorot_uniform')\n",
    "\n",
    "        self.W_regularizer = regularizers.get(W_regularizer)\n",
    "        self.b_regularizer = regularizers.get(b_regularizer)\n",
    "\n",
    "        self.W_constraint = constraints.get(W_constraint)\n",
    "        self.b_constraint = constraints.get(b_constraint)\n",
    "\n",
    "        self.bias = bias\n",
    "        self.step_dim = step_dim\n",
    "        self.features_dim = 0\n",
    "        super(Attention, self).__init__(**kwargs)\n",
    "\n",
    "    def build(self, input_shape):\n",
    "        assert len(input_shape) == 3\n",
    "\n",
    "        self.W = self.add_weight((input_shape[-1],),\n",
    "                                 initializer=self.init,\n",
    "                                 name='{}_W'.format(self.name),\n",
    "                                 regularizer=self.W_regularizer,\n",
    "                                 constraint=self.W_constraint)\n",
    "        self.features_dim = input_shape[-1]\n",
    "\n",
    "        if self.bias:\n",
    "            self.b = self.add_weight((input_shape[1],),\n",
    "                                     initializer='zero',\n",
    "                                     name='{}_b'.format(self.name),\n",
    "                                     regularizer=self.b_regularizer,\n",
    "                                     constraint=self.b_constraint)\n",
    "        else:\n",
    "            self.b = None\n",
    "\n",
    "        self.built = True\n",
    "\n",
    "    def compute_mask(self, input, input_mask=None):\n",
    "        # do not pass the mask to the next layers\n",
    "        return None\n",
    "\n",
    "    def call(self, x, mask=None):\n",
    "        # eij = K.dot(x, self.W) TF backend doesn't support it\n",
    "\n",
    "        # features_dim = self.W.shape[0]\n",
    "        # step_dim = x._keras_shape[1]\n",
    "\n",
    "        features_dim = self.features_dim\n",
    "        step_dim = self.step_dim\n",
    "\n",
    "        eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)), K.reshape(self.W, (features_dim, 1))), (-1, step_dim))\n",
    "\n",
    "        if self.bias:\n",
    "            eij += self.b\n",
    "\n",
    "        eij = K.tanh(eij)\n",
    "\n",
    "        a = K.exp(eij)\n",
    "\n",
    "        # apply mask after the exp. will be re-normalized next\n",
    "        if mask is not None:\n",
    "            # Cast the mask to floatX to avoid float64 upcasting in theano\n",
    "            a *= K.cast(mask, K.floatx())\n",
    "\n",
    "        # in some cases especially in the early stages of training the sum may be almost zero\n",
    "        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n",
    "\n",
    "        a = K.expand_dims(a)\n",
    "        weighted_input = x * a\n",
    "    #print weigthted_input.shape\n",
    "        return K.sum(weighted_input, axis=1)\n",
    "\n",
    "    def compute_output_shape(self, input_shape):\n",
    "        #return input_shape[0], input_shape[-1]\n",
    "        return input_shape[0],  self.features_dim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_attention_model1():\n",
    "    inp = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')\n",
    "    embedded_sequences = embedding_layer(inp)\n",
    "    x = Bidirectional(LSTM(256, dropout=0.25, recurrent_dropout=0.25, return_sequences=True))(embedded_sequences)\n",
    "    merged = Attention(MAX_SEQUENCE_LENGTH)(x)\n",
    "    merged = Dense(256, activation='relu')(merged)\n",
    "    merged = Dropout(0.25)(merged)\n",
    "    merged = BatchNormalization()(merged)\n",
    "    preds = Dense(1, activation='linear')(merged)\n",
    "    model = Model(inputs=inp, outputs=preds)\n",
    "    model.compile(loss='mse',optimizer='adam')\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_attention_model2():\n",
    "    inp = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')\n",
    "    embedded_sequences = embedding_layer(inp)\n",
    "    #Bidirectional LSTM  双向循环神经网络     \n",
    "    x = Bidirectional(LSTM(100, dropout=0.25, recurrent_dropout=0.25, return_sequences=True))(embedded_sequences)\n",
    "    merged = Attention(MAX_SEQUENCE_LENGTH)(x)\n",
    "    merged = Dense(100, activation='relu')(merged)\n",
    "    merged = Dropout(0.25)(merged)\n",
    "    \n",
    "#     merged = BatchNormalization()(merged)\n",
    "    merged = Attention(MAX_SEQUENCE_LENGTH)(x)\n",
    "    merged = Dense(50, activation='relu')(merged)\n",
    "    merged = Dropout(0.25)(merged)\n",
    "    \n",
    "    merged = BatchNormalization()(merged)\n",
    "    preds = Dense(1, activation='linear')(merged)\n",
    "    model = Model(inputs=inp, outputs=preds)\n",
    "    model.compile(loss='mse',optimizer='adam')\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def train_attention_model(model):\n",
    "    model_path = CACHE_PATH + \"attention_weights_best.hdf5\"\n",
    "    early = EarlyStopping(monitor=\"val_loss\", mode=\"min\", patience=5)\n",
    "    checkpoint = ModelCheckpoint(model_path, monitor='val_loss', verbose=1, save_best_only=True, mode='min')\n",
    "    callbacks_list = [checkpoint, early]\n",
    "    model.fit(X_train, y_train, batch_size=128, epochs=100, validation_data=(X_val, y_val), callbacks=callbacks_list)\n",
    "    model.load_weights(model_path)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = get_attention_model2()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.utils.vis_utils import plot_model\n",
    "plot_model(model,to_file='Atttention.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         (None, 300)               0         \n",
      "_________________________________________________________________\n",
      "embedding_1 (Embedding)      (None, 300, 300)          6000000   \n",
      "_________________________________________________________________\n",
      "bidirectional_2 (Bidirection (None, 300, 200)          320800    \n",
      "_________________________________________________________________\n",
      "attention_4 (Attention)      (None, 200)               500       \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 50)                10050     \n",
      "_________________________________________________________________\n",
      "dropout_4 (Dropout)          (None, 50)                0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_3 (Batch (None, 50)                200       \n",
      "_________________________________________________________________\n",
      "dense_6 (Dense)              (None, 1)                 51        \n",
      "=================================================================\n",
      "Total params: 6,331,601\n",
      "Trainable params: 331,501\n",
      "Non-trainable params: 6,000,100\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 209000 samples, validate on 11000 samples\n",
      "Epoch 1/100\n",
      "209000/209000 [==============================] - 981s 5ms/step - loss: 1.8074 - val_loss: 0.6180\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.61800, saving model to ../cache/attention_weights_best.hdf5\n",
      "Epoch 2/100\n",
      "209000/209000 [==============================] - 978s 5ms/step - loss: 0.4645 - val_loss: 0.4386\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.61800 to 0.43860, saving model to ../cache/attention_weights_best.hdf5\n",
      "Epoch 3/100\n",
      "209000/209000 [==============================] - 979s 5ms/step - loss: 0.4605 - val_loss: 0.8069\n",
      "\n",
      "Epoch 00003: val_loss did not improve\n",
      "Epoch 4/100\n",
      "209000/209000 [==============================] - 979s 5ms/step - loss: 0.4486 - val_loss: 0.8221\n",
      "\n",
      "Epoch 00004: val_loss did not improve\n",
      "Epoch 5/100\n",
      "209000/209000 [==============================] - 979s 5ms/step - loss: 0.4403 - val_loss: 1.2462\n",
      "\n",
      "Epoch 00005: val_loss did not improve\n",
      "Epoch 6/100\n",
      "209000/209000 [==============================] - 979s 5ms/step - loss: 0.4363 - val_loss: 0.4374\n",
      "\n",
      "Epoch 00006: val_loss improved from 0.43860 to 0.43736, saving model to ../cache/attention_weights_best.hdf5\n",
      "Epoch 7/100\n",
      "209000/209000 [==============================] - 979s 5ms/step - loss: 0.4290 - val_loss: 0.4386\n",
      "\n",
      "Epoch 00007: val_loss did not improve\n",
      "Epoch 8/100\n",
      "209000/209000 [==============================] - 978s 5ms/step - loss: 0.4233 - val_loss: 0.4458\n",
      "\n",
      "Epoch 00008: val_loss did not improve\n",
      "Epoch 9/100\n",
      "209000/209000 [==============================] - 978s 5ms/step - loss: 0.4221 - val_loss: 0.4142\n",
      "\n",
      "Epoch 00009: val_loss improved from 0.43736 to 0.41420, saving model to ../cache/attention_weights_best.hdf5\n",
      "Epoch 10/100\n",
      "209000/209000 [==============================] - 979s 5ms/step - loss: 0.4178 - val_loss: 0.4116\n",
      "\n",
      "Epoch 00010: val_loss improved from 0.41420 to 0.41160, saving model to ../cache/attention_weights_best.hdf5\n",
      "Epoch 11/100\n",
      "209000/209000 [==============================] - 978s 5ms/step - loss: 0.4152 - val_loss: 0.4136\n",
      "\n",
      "Epoch 00011: val_loss did not improve\n",
      "Epoch 12/100\n",
      "209000/209000 [==============================] - 979s 5ms/step - loss: 0.4105 - val_loss: 0.4130\n",
      "\n",
      "Epoch 00012: val_loss did not improve\n",
      "Epoch 13/100\n",
      "209000/209000 [==============================] - 978s 5ms/step - loss: 0.4078 - val_loss: 0.4215\n",
      "\n",
      "Epoch 00013: val_loss did not improve\n",
      "Epoch 14/100\n",
      "209000/209000 [==============================] - 978s 5ms/step - loss: 0.4048 - val_loss: 0.4241\n",
      "\n",
      "Epoch 00014: val_loss did not improve\n",
      "Epoch 15/100\n",
      "209000/209000 [==============================] - 979s 5ms/step - loss: 0.4030 - val_loss: 0.4125\n",
      "\n",
      "Epoch 00015: val_loss did not improve\n"
     ]
    }
   ],
   "source": [
    "model = train_attention_model(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "50000/50000 [==============================] - 58s 1ms/step\n"
     ]
    }
   ],
   "source": [
    "y_test = model.predict(X_test,batch_size=128,verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def round_score(score):\n",
    "    if score>5:\n",
    "        score = 5\n",
    "    if  score < 1:\n",
    "        score= 1\n",
    "    return score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sub = pd.read_csv(INPUT_PATH + 'sample.csv',header=None,names=['Id','Score'])\n",
    "sub['Score'] = y_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sub['Score'] = sub['Score'].map(round_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7fd74e95f160>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXsAAAD6CAYAAABApefCAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAD91JREFUeJzt3V+MnNdZx/HvUydBxktdR45WJBe1\nIllIEY7beAl2ccNslFgJhgoCUgumVdQGSxB6QXJRVwl/ipLKqpoIpTSgFaFElWDlgBraOsGhyEus\nkkCzDYkrJFT+bFANaamS2t2oFyR6uJjX8Wa9u/PO7L7zzu75fqSVZ86cmXnmePY3Z8/7ZyIzkSRt\nbG9ruwBJUvMMe0kqgGEvSQUw7CWpAIa9JBXAsJekAhj2klQAw16SCmDYS1IBLmm7gPO2b9+eO3bs\nGOi+r732Glu2bFnbgtbAqNYFo1ubdfXHuvqzEeuanZ39bmZe0bNjZo7Ez549e3JQJ0+eHPi+TRrV\nujJHtzbr6o919Wcj1gU8lzUy1mUcSSqAYS9JBTDsJakAhr0kFcCwl6QCGPaSVADDXpIKYNhLUgEM\ne0kqwMicLkGShm3HkeNvXp47erDFSprnzF6SCmDYS1IBDHtJKoBhL0kFMOwlqQCGvSQVwLCXpAIY\n9pJUAMNekgpg2EtSAQx7SSqAYS9JBTDsJakAhr0kFcCwl6QCGPaSVADDXpIKUCvsI2I8Ip6vLj8S\nEc9ExL0Lbq/VJklqR92Z/aeBzRFxG7ApM/cBV0fEzrptzZQvSaqjZ9hHxI3Aa8DLQAc4Vt30FLC/\njzZJUksiM5e/MeIy4ATwC8DjwL8DD2XmCxFxALgO2FmnLTOPLvH4h4HDAOPj43ump6cHehHz8/OM\njY0NdN8mjWpdMLq1WVd/rKs/i+s6febsm5d3XbW1jZKA1Y3X5OTkbGZO9Op3SY/bjwAPZ+b3IgJg\nHthc3TZG9y+Dum0XycwpYApgYmIiO51Or3qXNDMzw6D3bdKo1gWjW5t19ce6+rO4rtuPHH/z8tyh\nzsV3GJJhjFevZZybgDsjYgZ4F/BzXFiS2Q3MAbM12yRJLVlxZp+ZN5y/XAX++4BTEXElcCuwF8ia\nbZKkltTezz4zO5l5ju7G12eBycw8W7dtrQuXJNXXa83+Ipn5Khf2tOmrTZLUDo+glaQCGPaSVADD\nXpIKYNhLUgEMe0kqgGEvSQUw7CWpAIa9JBXAsJekAhj2klSAvk+XIEkb3Y6Fpz4+erDFStaOM3tJ\nKoBhL0kFMOwlqQCGvSQVwLCXpAIY9pJUAMNekgrgfvaSxFv3rd+InNlLUgEMe0kqgGEvSQUw7CWp\nAIa9JBXAsJekAhj2klQAw16SCmDYS1IBPIJWklawUb61ypm9JBXAsJekAhj2klQAw16SCmDYS1IB\nDHtJKoBhL0kFMOwlqQC1wj4iLo+ImyNie9MFSZLWXs8jaCNiG/Bl4DjwYETcCBwFrgGOZ+Z9Vb9H\n6rRJ0nq1+Htq19MRtXVm9tcCd2Xm/cAJ4EZgU2buA66OiJ0RcVudtqZehCRpZZGZ9TpG3ADcB/wb\n8JeZ+UREfADYDLwb+JtebZn5uUWPeRg4DDA+Pr5nenp6oBcxPz/P2NjYQPdt0qjWBaNbm3X1x7r6\ns7iu02fOrurxdl21dbUlAasbr8nJydnMnOjVr9aJ0CIigPcDrwIJnKluegW4DthSs+0tMnMKmAKY\nmJjITqdTp5yLzMzMMOh9mzSqdcHo1mZd/bGu/iyu6/ZFyzL9mjvU6dmnjmGMV60NtNl1J/Ai8B66\nM3eAseox5mu2SZJa0DOAI+JjEfGh6uo76G6c3V9d3w3MAbM12yRJLaizjDMFHIuIO4BvAI8DT0fE\nlcCtwF66SzunarRJklrQM+wz81Xg5oVtEdGp2j6VmWf7aZMkDd9A31RVfQAcG6RNkjR8bjSVpAIY\n9pJUAMNekgpg2EtSAQx7SSqAYS9JBTDsJakAhr0kFcCwl6QCGPaSVADDXpIKYNhLUgEMe0kqgGEv\nSQUw7CWpAIa9JBXAsJekAhj2klQAw16SCmDYS1IBDHtJKoBhL0kFMOwlqQCGvSQV4JK2C5Ck9WrH\nkeNvXp47erDFSnpzZi9JBXBmL6koC2fjJTHsJW145wP+7l2vU2rsuYwjSQUw7CWpAIa9JBXAsJek\nAhj2klQAw16SClDmPkiSNrxS96dfjjN7SSqAYS9JBegZ9hGxNSKejIinIuILEXFZRDwSEc9ExL0L\n+tVqkyQNX52Z/SHgwcw8ALwMfADYlJn7gKsjYmdE3FanrakXIUlaWc8NtJn58IKrVwC/CvxBdf0p\nYD/wbuBYjbZvrr5kSVK/IjPrdYzYB9wHzAEPZeYLEXEAuA7YWactM48ueszDwGGA8fHxPdPT0wO9\niPn5ecbGxga6b5NGtS4Y3dqsqz/WtbzTZ85e1Da+Gb79g2aeb9dVWwe+72rGa3JycjYzJ3r1q7Xr\nZURcDnwG+EXgLmBzddMY3aWg+Zptb5GZU8AUwMTERHY6nTrlXGRmZoZB79ukUa0LRrc26+qPdS3v\n9iV2vbx71+s8cLqZPc7nDnUGvu8wxqvOBtrLgMeAj2fmS8As3SUZgN10Z/p12yRJLajzEfcRussy\n90TEPcDngA9GxJXArcBeIIFTNdokSS3oObPPzD/KzG2Z2al+HgU6wLPAZGaezcxzddqaehGSpJUN\ntHiVma9yYU+bvtokScPnuXEkaQ0sPBfP3NGDLVayNE+XIEkFMOwlqQCGvSQVwLCXpAIY9pJUAMNe\nkgpg2EtSAdzPXtKG4ffOLs+ZvSQVwLCXpAIY9pJUANfsJa1rrtPX48xekgpg2EtSAQx7SSqAYS9J\nBTDsJakAhr0kFcCwl6QCGPaSVADDXpIKYNhLUgEMe0kqgGEvSQUw7CWpAJ71UtK645ku++fMXpIK\nYNhLUgEMe0kqgGEvSQUw7CWpAIa9JBXAsJekAhj2klQAD6qStC54INXqOLOXpAIY9pJUgFphHxHj\nEXGqunxpRHwpIr4aER/up02S1I6ea/YRsQ14FNhSNX0UmM3M34uIJyLiMeDX6rRl5vebeiGSNh7X\n6ddOZObKHSLeDgTw15nZiYgvAkcy818i4gjwj8Bv1WnLzJOLHvswcBhgfHx8z/T09EAvYn5+nrGx\nsYHu26RRrQtGtzbr6s9Gr+v0mbNrUM0F45vh2z9Y04dc0q6rtvbVfzXjNTk5OZuZE7369ZzZZ+Y5\ngIg437QFOFNdfgUY76Nt8WNPAVMAExMT2el0epWzpJmZGQa9b5NGtS4Y3dqsqz8bpa7lZ/Bru8Pg\n3bte54HTze+EOHeo01f/Yfw/DrKBdh7YXF0eqx6jbpskqQWDBPAssL+6vBuY66NNktSCQf6eeRR4\nIiLeC1xDd33+TM02SVILaod9Znaqf1+KiJvpztp/JzPfAOq2SRLgnjbDNtCWisz8b+DYIG2SpOFz\no6kkFcAToUlq1MLlmrmjB1uspGyGvaShcZ2+PS7jSFIBnNlLWnOnz5zldmfxI8Wwl6Q1NorbKVzG\nkaQCGPaSVADDXpIKYNhLUgHcQCtpTSzcKHn3rhYL0ZKc2UtSAQx7SSqAyziSBubpD9YPw15ST6N4\nkJD6Y9hL6ouz+fXJNXtJKoBhL0kFcBlHKkyd9XeXajYeZ/aSVABn9pIAZ/MbnTN7SSqAYS9JBXAZ\nRyqYSzfNG5UD0gx7qQCGulzGkaQCOLOXNpBRWTJQbwv/r/7sli2NP59hL21QLt1oIcNeWucMddXh\nmr0kFcCZvbROLDWDv3vX6/hrrDqc2UtSAZwSSCPGNfiNq83/W8NeGiKDXG0x7KWGGfAaBa7ZS1IB\nnNlLy1juCMfljlJ1Bq9R1njYR8QjwDXA8cy8r+nnk1ZjucA+feYsty9xmwGv9aLRsI+I24BNmbkv\nIv40InZm5jebfE6tf35HqrT2mp7Zd4Bj1eWngP3Amof9wlnXqJ/8aTUnquo3BFd6/OXGrO0QXXyQ\nUNv1SBtFZGZzD95dwnkoM1+IiAPAdZl5dMHth4HD1dUfA/51wKfaDnx3VcU2Y1TrgtGtzbr6Y139\n2Yh1vTMzr+jVqemZ/Tywubo8xqK9fzJzCpha7ZNExHOZObHax1lro1oXjG5t1tUf6+pPyXU1vevl\nLN2lG4DdwFzDzydJWkLTM/vHgVMRcSVwK7C34eeTJC2h0Zl9Zp6ju5H2WWAyM8829FSrXgpqyKjW\nBaNbm3X1x7r6U2xdjW6glSSNBk+XII24iLg8Im6OiO1t16L1y7BfIxExHhGnVrj9qoj4VkTMVD89\nd5XaiCJia0Q8GRFPRcQXIuKyJfpcEhH/tWCsdg2ptpEL1YjYBnwZuB44udT7pq3xGmXV7+Pzy9xW\n5Hitu7CvEaqXRsSXIuKrEfHhIdW0DXgUWOkr4n8SuD8zO9XP/zZcU89Qrfo9EhHPRMS9TdazwCHg\nwcw8ALwM3LJEn2uBv1gwVqebLqpOqFb9hj1e1wJ3Zeb9wAngumX6DHW8zlspVKvbhz1e532aC7t9\nL9bG+6vWB0xEfCIivhYRn13rGtZV2NcM1Y8Cs5n5U8AvRcSPDKG0N4D3A+dW6LMXuCMivh4RnxxC\nTT1DdeHpLICrI2Jn00Vl5sOZ+bfV1SuA7yzRbS/wsxHxT1VYDOOEfT1DtaXx+vvMfDYibqD7QfTM\nEt3aGK/zlg3VNsaret4bgdfovu+X0tb7a8UPmIjYQ3dX9euB70TETWtZwLoKe+qFaocLp2h4Gmj8\nAIrMPFdjT6Mn6db2E8C+iLi24ZrqhGqHi09nMRQRsQ/YlpnPLnHz14CbMvN64FLgZ5qup2aodmhh\nvCIi6L7vXwX+b4kuQx+vqq5eodphyONV/QX728CRFbq1MV51PmB+Gvir7O41cwJ471oWsK7Cvmao\nbgHOVJdfAcabraq2f8jM72fmG8DzwLBmOSuFaitjFRGXA58BlltmezEz/6e6/BzDG6teodrKeGXX\nncCLwPuW6DL08aoZqm2M1xHg4cz83gp92nh/1fmAaXS81lXY17TiKRpadCIifjQifhg4AHyj6Ses\nEapDH6sqJB4DPp6ZLy3T7fMRsTsiNgE/D7zQdF1QK1TbGK+PRcSHqqvvAJYKsTbGq06otvG7eBNw\nZ0TMAO+KiD9Zok8b41XnA6bR8RqVIFxLrZ+iISJujIjfXNT8CeAk3QPM/jgzBz3pW90a6oRqG2P1\nEbrr4fdUG6p+NyIWf8/B7wOfB/4ZeCYzv9J0UTVDtY3xmgI+GBFPA5uAb43CeFEvVIc+Xpl5w/l1\ncbrj8eCIjFedD5hGx2tdHlQVETOZ2anWDK/JzD9ccNs7gSeArwDvAfZWSydFiYhfBz7JhTfVSeDS\nzLx3QZ+3A6eAv6M6nUWDRzmPtGrj/zHgh+j+1fVZ4Jcdr96qwP8N4Fccr6VFxI8Dfw4E8EXgAeBT\nmXnHgj5voztez9HdoeKWzPzPNathPYZ9L9E9F89+4ESpb666qpC7GXg6M5fb0KaK49Ufx6s/EbEZ\nOAh8PTP/Y00feyOGvSTprTbimr0kaRHDXpIKYNhLUgEMe0kqgGEvSQX4fy5i0Z5DNwijAAAAAElF\nTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fd74c5dbf60>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "sub['Score'].hist(bins = 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sub.to_csv(OUTPUT_PATH + 'attention_0404.csv',index=False, header=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
